# -*- coding = utf-8 -*-
# @Time    : 2025/3/13 下午6:36
# @Author  : yqk
# @File    : 爬虫基础-豆瓣.py
# @Software: PyCharm



from bs4 import BeautifulSoup  # 网页解析
import re
import requests  # 修正模块名
import xlwt  # 用于保存数据为Excel
import sqlite3  # 用于保存数据到数据库


def main():
    baseurl = "https://movie.douban.com/top250?start="
    getData(baseurl)
  
    # savepath = ".\\豆瓣电影Top250.xls"
    # saveData(datalist, savepath)


findlink = re.compile(r'<a href="(.*?)">')
findImg = re.compile(r'<img.* src="(.*?)"',re.S)
findtitle = re.compile(r'<span class="title">(.*?)</span>')
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')  # 提取评分
findJudge = re.compile(r'<span>(\d*)人评价</span>')  # 提取评价人数
findIng = re.compile(r'<span class="other">(.*?)</span>')   # 提取影片相关内容
findBd = re.compile(r'<div class="bd">\s*<p>(.*?)</p>', re.S) # 提取影片相关内容

def clean_data(data):
    # 去除 HTML 标签
    data = re.sub(r'<.*?>', '', data)

    data = data.replace("\xa0", " ").replace("\n", " ").replace("<br/>", " ").strip()

    data = re.sub(r'\s+', ' ', data)
    return data


def getData(baseurl):
    headers = {
        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "accept-language": "zh-CN,zh;q=0.9",
        "referer": "https://movie.douban.com/top250?start=225&filter=",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"
    }

    datalist = []
    for i in range(0, 1):  # 这里只抓取第一页，实际使用时可以调整
        url = baseurl + str(i * 25)

        # 爬取页面内容
        response = requests.get(url, headers=headers)
        # 解析页面内容
        soup = BeautifulSoup(response.content.decode('utf-8'), 'html.parser')

        for item in soup.find_all('div', class_='item'):
            data = []
            item = str(item)

            link = re.findall(findlink, item)
            img = re.findall(findImg, item)
            title = re.findall(findtitle, item)
            rating = re.findall(findRating, item)
            judge = re.findall(findJudge, item)
            ing = re.findall(findIng, item)
            bd = re.findall(findBd, item)

            # 数据处理
            data.append(clean_data(link[0]) if link else "无链接")  # 链接
            data.append(clean_data(img[0]) if img else "无图片")  # 图片链接
            data.append(clean_data(title[0]) if title else "无标题")  # 电影标题
            data.append(clean_data(rating[0]) if rating else "无评分")  # 评分
            data.append(clean_data(judge[0]) if judge else "无评价")  # 评价人数
            data.append(clean_data(ing[0]) if ing else "暂无信息")  # 正在上映等信息
            data.append(clean_data(bd[0]) if bd else "暂无描述")  # 电影简介

            datalist.append(data)  # 添加到总数据列表

    # 打印结果
    for data in datalist:
        print(data)


    return datalist


def saveData(datalist, savepath):
    print("正在保存数据...")


if __name__ == '__main__':
    main()





